# -*- coding: utf-8 -*-
import requests
import datetime
import json
import time
import multiprocessing
import MySQLdb
import urllib2
import ssl
import redis
import sys
import pytz

from elasticsearch import Elasticsearch, helpers

ssl._create_default_https_context = ssl._create_unverified_context

# 连接redis
r = redis.Redis(host='127.0.0.1', port=6379, decode_responses=True)

es = Elasticsearch(['10.201.241.41', '10.201.241.42', '10.201.241.43'], sniff_on_start=True, sniff_on_connection_fail=True, max_retries=3, retry_on_timeout=True)


# 本地时间转为utc时间
def local_to_utc(local_ts, utc_format='%Y-%m-%dT%H:%M:%S.000'):
    local_tz = pytz.timezone('Asia/Chongqing')
    local_format = "%Y-%m-%d %H:%M:%S"
    time_str = time.strftime(local_format, time.localtime(local_ts))
    dt = datetime.datetime.strptime(time_str, local_format)
    local_dt = local_tz.localize(dt, is_dst=None)
    utc_dt = local_dt.astimezone(pytz.utc)
    return utc_dt.strftime(utc_format)


class ES(object):
    @classmethod
    def connect_host(cls):
        hosts = [{"host": "10.201.240.1"},
                 {"host": "10.201.240.2"},
                 {"host": "10.201.240.3"},
                 {"host": "10.201.240.7"},
                 {"host": "10.201.240.12"}]
        es = Elasticsearch(hosts, sniff_on_start=True,
                           sniff_on_connection_fail=True,
                           sniff_timeout=60)
        return es


class GetData(object):

    def __init__(self, index, query, aggs={}, timestart=None, timeend=None):
        self.index = index
        self.query = query
        self.timestart = timestart
        self.timeend = timeend
        self.aggs = aggs
        self.d = {}
        self.p = {}
        self.es = ES.connect_host()

    def _istimeformat(self, date):
        if ":" in date:
            if len(date.split(":")) == 3:
                if time.strptime(date, '%Y-%m-%d %H:%M:%S'):
                    return date
                else:
                    return False
            elif len(date.split(":")) == 2:
                if time.strptime(date, '%Y-%m-%d %H:%M'):
                    return date + ":00"
                else:
                    return False
        else:
            if time.strptime(date, '%Y-%m-%d'):
                return date.strip(" ") + " 00:00:00"
            else:
                return False

    def get_time(self):
        if self.timestart is not None and self.timeend is not None:
            t1 = self._istimeformat(self.timestart)
            t2 = self._istimeformat(self.timeend)
            if t1:
                st = int(time.mktime(time.strptime(t1, "%Y-%m-%d %H:%M:%S")))
                if t2:
                    et = int(time.mktime(time.strptime(t2, "%Y-%m-%d %H:%M:%S")))
                    if st > et:
                        print "开始时间超过结束时间。"
                        sys.exit(2)
                    else:
                        return st, et
                else:
                    print "结束时间格式不对，请检查后再用"
                    sys.exit(1)
            else:
                print "开始时间格式不对"
                sys.exit(1)
        elif self.timestart is not None and self.timeend is None:
            t1 = self._istimeformat(self.timestart)
            temp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            t2 = self._istimeformat(temp)
            if t1:
                st = int(time.mktime(time.strptime(t1, "%Y-%m-%d %H:%M:%S")))
                et = int(time.mktime(time.strptime(t2, "%Y-%m-%d %H:%M:%S")))
                return st, et
            else:
                print "开始时间格式不对"
                sys.exit(1)
        elif self.timestart is None and self.timeend is None:
            et = datetime.datetime.now()
            end_time = et.strftime("%Y-%m-%d %H:%M")
            end_time = end_time + ":00"
            if '499' in self.query:
                ms = 1
            else:
                ms = 3
            start_time = (datetime.datetime.now() - datetime.timedelta(minutes=ms)).strftime("%Y-%m-%d %H:%M")
            start_time = start_time + ":00"
            st = int(time.mktime(time.strptime(start_time, "%Y-%m-%d %H:%M:%S")))
            et = int(time.mktime(time.strptime(end_time, "%Y-%m-%d %H:%M:%S")))
            return st, et
        else:
            print "时间格式不对"
            sys.exit(3)

    def get_body(self, aggs={}):
        start_time, end_time = self.get_time()
        start_time = str(start_time) + '000'
        end_time = str(end_time) + '000'
        body = {
            "size": 0,
            "query": {
                "filtered": {
                    "query": {
                        "query_string": {
                            "query": self.query,
                            "analyze_wildcard": "true"
                        }
                    },
                    "filter": {
                        "bool": {
                            "must": [{
                                "range": {
                                    "timestamp": {
                                        "gte": start_time,
                                        "lte": end_time,
                                        "format": "epoch_millis"
                                    }
                                }
                            }],
                            "must_not": []
                        }
                    }
                }
            },
            "aggs": aggs
        }
        return body

    def get_total(self):
        total_boby = self.get_body()
        res_total = self.es.search(index=self.index, body=total_boby)
        total_num = res_total['hits']['total']
        return total_num

    def get_data(self):
        c_body = self.get_body(self.aggs)
        res = self.es.search(index=self.index, body=c_body)
        return res


# 获得cpu, memory
class GetCpuMem:

    nodenum = 0

    def __init__(self, env, appname, appid, times):
        self.env = env
        self.appname = appname
        self.appid = appid
        self.times = times
        self.ips = self.getips()
        self.start_time, self.end_time = self.get_time()

    # 获得应用所有节点
    def getips(self):
        templist = []
        sql = "select bus_ip from app_node where db_status=1 and app_basis_an_id=%s" % self.appid
        data = exec_sql(sql)
        if data:
            for i in data:
                if i[0] not in templist:
                    templist.append(i[0])
		self.nodenum = self.nodenum + 1
        return templist

    # 获得对应的字段的url
    def geturl(self, urlstr):
        url_dict = {
            '生产环境': 'http://prometheus3.blops.com:9090/api/v1/query_range?query=',
            '仿真环境': 'http://prometheus2.blops.com:9090/api/v1/query_range?query='
        }
        url = "%s%s&start=%s&end=%s&step=1m" % (url_dict[str(self.env)], urlstr, self.start_time, self.end_time)
        return url

    # 获得开始和结束时间
    def get_time(self):
        end_time = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(self.times-(8*3600))) + '.000Z'
        start_time = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(self.times - 300 - (8*3600))) + '.000Z'
        return start_time, end_time

    # 获得查询条件
    def get_query_str(self, ip, field):
        avg_str, max_str = '', ''
        if field == 'cpu':
            avg_str = '100 - avg_over_time(cpu_usage_idle{instance="%s", cpu="cpu-total"}[1m])' % ip
            max_str = '100 - min_over_time(cpu_usage_idle{instance="%s", cpu="cpu-total"}[1m])' % ip
        elif field == 'mem':
            max_str = 'max_over_time(mem_used_percent{instance="%s"}[1m])' % ip
            avg_str = 'avg_over_time(mem_used_percent{instance="%s"}[1m])' % ip
        return max_str, avg_str

    # 通过url获得数据
    def get_data(self, url):
        total = 0.0
        count = 0
        res = ''
	while res == '':
            try:
                res = requests.get(url)
                if res.status_code == 200:
                    data = res.json()
                    if 'status' in data.keys() and data['status'] == 'success':
                        if data['data']['result'] and data['data']['result'][0]['values']:
                                for i in data['data']['result'][0]['values']:
                                    count = count + 1
                                    total = total + float(i[1])
            except Exception as e:
                print '访问太多需要等一会儿'
                print 'sleep 3'
                time.sleep(3)
                continue
        if count == 0:
            return total
        else:
            return total / count

    # 获得每个ip的数据
    def get_prometheus_data(self, ip, field):
        value_max, value_avg = 0, 0
        max_str, avg_str = self.get_query_str(ip, field)
        max_url = self.geturl(max_str)
        avg_url = self.geturl(avg_str)
        value_max = self.get_data(max_url)
        value_avg = self.get_data(avg_url)
        return value_max, value_avg

    # 获得字段聚合后的数据
    def getfielddata(self, field):
        max_value, avg_value = 0, 0
        maxlist = []
        avglist = []
        if len(self.ips) > 0:
            for ip in self.ips:
                amax, aavg = self.get_prometheus_data(ip, field)
                maxlist.append(amax)
                avglist.append(aavg)
            if maxlist:
                max_value = round(sum(maxlist)/len(maxlist), 3)
            if avglist:
                avg_value = round(sum(avglist)/len(avglist), 3)
        return max_value, avg_value


# 获得TPS
class GetTps:

    def __init__(self, appname, times):
        self.appname = appname
        self.times = times

    def gen_dates(self, b_date, days):
        day = datetime.timedelta(days=1)
        for i in range(days):
            yield b_date + day * i

    def get_time(self):
        end_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.times))
        start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.times - 300))
        return start_time, end_time

    def hanld_data(self, data):
        templist = []
        vmax, vavg = 0.0, 0.0
        if 'aggregations' in data.keys():
            if 'data' in data['aggregations']:
                if 'buckets' in data['aggregations']['data'] and data['aggregations']['data']['buckets']:
                    for i in data['aggregations']['data']['buckets']:
                        templist.append(i['doc_count'])
        if templist:
            vmax = round(max(templist)/5, 3)
            vavg = round(sum(templist)/len(templist)/5, 3)

        return vmax, vavg

    def formatdata(self):
        st, et = self.get_time()
        tpsmax, tpsavg, qpsmax, qpsavg = 0.0, 0.0, 0.0, 0.0
        index = 'graylog*'
        query_tps = 'ngx_tag:%s AND response:<400' % self.appname
        query_qps = 'ngx_tag:%s' % self.appname

        aggs = {
            'data': {
                "date_histogram": {
                    "field": "timestamp",
                    "interval": "5s",
                    "time_zone": "Asia/Shanghai",
                    "min_doc_count": 0,
                }
            }
        }
        res_tps = GetData(index, query_tps, aggs=aggs, timestart=st, timeend=et)
        if res_tps:
            tpsmax, tpsavg = self.hanld_data(res_tps.get_data())
        res_qps = GetData(index, query_qps, aggs=aggs, timestart=st, timeend=et)
        if res_qps:
            qpsmax, qpsavg = self.hanld_data(res_qps.get_data())
        return tpsmax, tpsavg, qpsmax, qpsavg


def get_prefive_time(times):
    startime = times + " 00:00:00"
    endtime = times + " 23:59:59"
    st = int(time.mktime(time.strptime(startime, '%Y-%m-%d %H:%M:%S')))
    et = int(time.mktime(time.strptime(endtime, '%Y-%m-%d %H:%M:%S')))

    return range(st, et, 300)


# 获得ElasticSearch 服务
def clusterhealth():
    es_server = [
        'http://10.201.241.41:9200',
        'http://10.201.241.42:9200',
        'http://10.201.241.43:9200'
    ]
    endpoint = '/_cluster/health'
    for es in es_server:
        url = es + endpoint
        try:
            res = requests.get(url)
            jsondata = res.json()
            if jsondata['status'] == 'green' or jsondata['status'] == 'yellow':
                return es
        except Exception as e:
            print e
            continue


# 写入ElasticSearch
def post_data(data, dates=None):
    elasticIndex = 'blbasedata-' + format(datetime.datetime.now(), '%Y-%m-%d')
    if dates is not None:
        elasticIndex = 'blbasedata-' + dates

    adict = [{"_index": elasticIndex , "_type": "message", "_source": data}]

    try:
        helpers.bulk(es, adict)
        return True
    except Exception as e:
        print("try again")
        return 0

def write_data(appname, appid, times):
    res = GetCpuMem('生产环境', appname, appid, times)
    cpumax, cpuavg = res.getfielddata('cpu')
    memorymax, memoryavg = res.getfielddata('mem')
    nodenum = res.nodenum 
    tps = GetTps(appname, times)
    tpsmax, tpsavg, qpsmax, qpsavg = tps.formatdata()
    tempdict = {
        'appname': appname,
        'tpsmax': tpsmax,
        'tpsavg': tpsavg,
        'cpumax': cpumax,
        'cpuavg': cpuavg,
        'memorymax': memorymax,
        'memoryavg': memoryavg,
        'nodenum': nodenum,
        'qpsmax': qpsmax,
        'qpsavg': qpsavg,
        '@timestamp': local_to_utc(times)
    }
    dates = time.strftime('%Y-%m-%d', time.localtime(times))
    res = post_data(tempdict, dates=dates)
    return res


# 使用多进程来获得各工程数据
def get_appname_data(applist, times):
    results = []
    pre_pool = multiprocessing.Pool(processes=100, maxtasksperchild=1000)
    n = 0
    for appid, appname in applist:
    #     if appname == 'member-service':
    #         res = write_data(appname, appid, times)
    #         results.append(res)
        n = n + 1
        print u'----工程--%s %s' % (n, appname)
        res = pre_pool.apply_async(write_data, args=(appname, appid, times))
    # res = write_data(appname, appid, times)
        results.append(res)
    pre_pool.close()
    pre_pool.join()

    return results

# 执行sql
def exec_sql(sql):
    templist = []
    db = MySQLdb.connect('10.201.241.17','blcmdb', 'B|9l8C7m6D5|b', 'blcmdb', charset='utf8')
    cursor = db.cursor()
    try:
        cursor.execute(sql)
        data = cursor.fetchall()
        if data:
            for i in data:
                templist.append(i)
    except Exception as e:
        print e
        return None
    else:
       db.close()
       return templist

# 获得工程列表
def get_appid_appname_list():
    sql = "select id, app_name from app_basis where db_status=1 and environment='生产环境'"
    data = exec_sql(sql)
    return data


if __name__ == '__main__':
    b = []
    a = ['2017-12-04', '2017-12-05', '2017-12-06', '2017-12-07', '2017-12-08', '2017-12-09', '2017-12-10', '2017-12-11', '2017-12-12', '2017-12-13', '2017-12-14', '2017-12-15', '2017-12-16', '2017-12-17', '2017-12-18', '2017-12-19', '2017-12-20', '2017-12-21', '2017-12-22', '2017-12-23', '2017-12-24', '2017-12-25', '2017-12-26', '2017-12-27', '2017-12-28', '2017-12-29', '2017-12-30', '2017-12-31', '2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04', '2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08', '2018-01-09', '2018-01-10', '2018-01-11', '2018-01-12', '2018-01-13', '2018-01-14', '2018-01-15', '2018-01-16', '2018-01-17', '2018-01-18', '2018-01-19', '2018-01-20', '2018-01-21', '2018-01-22', '2018-01-23', '2018-01-24', '2018-01-25', '2018-01-26', '2018-01-27', '2018-01-28', '2018-01-29', '2018-01-30', '2018-01-31', '2018-02-01', '2018-02-02', '2018-02-03', '2018-02-04', '2018-02-05', '2018-02-06', '2018-02-07', '2018-02-08', '2018-02-09', '2018-02-10', '2018-02-11', '2018-02-12', '2018-02-13', '2018-02-14', '2018-02-15', '2018-02-16', '2018-02-17', '2018-02-18', '2018-02-19', '2018-02-20', '2018-02-21', '2018-02-22', '2018-02-23', '2018-02-24', '2018-02-25', '2018-02-26', '2018-02-27', '2018-02-28', '2018-03-01']
    applist = get_appid_appname_list()
    a.reverse()

    for d in a:
        print u'开始------%s------' % d
        times = get_prefive_time(d)
        for i in times:
            res = get_appname_data(applist, i)
            b.append(res)
    print '--------------%s end--------------' % d
    print '===================='
